import requests
import re
from  bs4 import BeautifulSoup
import multiprocessing
import asyncio
import aiohttp
import time
from pymongo import MongoClient
conn = MongoClient(host='127.0.0.1',port=27017)
db = conn.myproxies
mset = db.proxy #ip集中于一个集合内，KEY是iP 然后VALUE是连接程度
proxyurl = 'http://www.xicidaili.com/nn/'
proxyurl2 = 'https://www.kuaidaili.com/free/inha/'

def proxytolist(proxyurl):
    proxyurlli = []
    for i in range(5):#5页后面大部分都用不了 做个5页就可以了
        proxyurlli.append(proxyurl+str(i+1))
    return proxyurlli
def  multicaozuo(proxyurl):
    successli = []
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'
               }
    data = requests.get(proxyurl,headers=headers)
    soup  = BeautifulSoup(data.text,'lxml')
    td   = soup.select('#ip_list')
    if td == []:
        td = soup.select('#list')
    data = ''
    for i in td:
        data=data + str(i.get_text())
    data = data.split("\n")
    patten =re.compile('\d+\.\d+\.\d+\.\d+')
    patten2 = re.compile('^\d{1,5}$')
    patten3 = re.compile('(HTT.*)')
    patten4 = re.compile('socks')
    ipli = []
    duankou = []
    xieyi = []

    for i in data:
        res1 = patten.findall(i)
        if res1==[]:
            res2 = patten2.findall(i)
            if res2 == []:
                res3 = patten3.findall(i)
                if res3 == []:
                    res4 = patten4.findall(i)
                    if res4!=[]:
                        xieyi.append('socks5')
                else:
                    xieyi.append(res3[0])
            else:
                duankou.append(res2[0])
        else:
            ipli.append(res1[0])
    ipzip = zip(ipli,duankou,xieyi)
    url = 'http://ip.chinaz.com/'
    # for i in ipzip:


    async  def ceshiip(session,proxy,ipdizhi,meth,Ip):
            try:
                async with session.get(url,headers=headers,proxy = proxy,timeout=15)as data:
                #     data = requests.get(url,headers=headers,proxies = proxy)
                    patten = re.compile('您来自：</span>(\d+\.\d+\.\d+\.\d+)')
                    html = await data.text()
                    res = patten.findall(html)
                    print (res[0])
                    if res[0] == Ip:
                        print ("验证成功 ip是%s方式是%s"%(ipdizhi,meth))
                        successli.append(proxy)
                        #做成key = ip+duankou value = 50  socks http https 方式暂时不做做成打分系统 默认是http 或者https
                        #先搜索有没有相同的key  有的话看不加入 没有的话加入并设置值为50
                        sec = mset.find({'ip':ipdizhi}).count()
                        print(sec)
                        if sec == 0:
                            mset.insert({'ip':ipdizhi,'value':50})

            except:
                print ("%s连接失败"%proxy)
            return 0

    async def yibumain(loop):
        async with aiohttp.ClientSession()as session:
            tasks = []
            for i in ipzip:
                ipdizhi =  i[0]+':'+i[1]
                meth    =  i[2]
                # proxy={
                #     'http' : meth.lower()+'://'+ipdizhi,
                #     'https': meth.lower()+'://'+ipdizhi
                # }
                proxy = 'http://'+ipdizhi
                proxy2 = 'https://'+ipdizhi
                task = loop.create_task(ceshiip(session,proxy,ipdizhi,meth,i[0]))
                tasks.append(task)
                task2 = loop.create_task(ceshiip(session,proxy2,ipdizhi,meth,i[0]))
                tasks.append(task2)
            if tasks != []:
                await asyncio.wait(tasks)

    loop = asyncio.get_event_loop()
    loop.run_until_complete(yibumain(loop))
    return successli


def multi(pool):

    proxyurlli = proxytolist(proxyurl)
    proxyurlli2 = proxytolist(proxyurl2)
    successpool = pool.map(multicaozuo,proxyurlli)
    successpool2 = pool.map(multicaozuo,proxyurlli2)
    successli = []
    for i in successpool:
        successli.extend(i)
    for i in successpool2:
        successli.extend(i)
    print (successli)
    return successli

def yanzheng(con,pool):
    # data = open('C:/Users/yehuo/PycharmProjects/proxy/a.txt')
    # data = data.read()
    # data = data.replace(']','')
    # data = data.replace('[','')
    # data = data.replace(' ','')
    # data = data.replace('\'','')
    # data = data.split(',')
    data=[]
    sss = mset.find({'ip':{'$ne':None}},{'_id':0,'ip':1})
    for i in sss:

        data.append(i['ip'])

    # for i in data:
    pool.map(yanzhengduojincheng, data)
    return 0


def yanzhengduojincheng(i):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'
    }
    url = 'http://ip.chinaz.com/'

    proxy = {
        'http':i,
        'https':i,
    }
    print (proxy)


    val = mset.find_one({'ip':i},{'_id':0,'value':1})
    val = val['value']

    val = int(val)
    try:
        html = requests.get(url,headers=headers,proxies = proxy,timeout=20)
        patten = re.compile('您来自：</span>(\d+\.\d+\.\d+\.\d+)')
        res = patten.findall(html.text)
        print (res)
        if res[0]!=[]:
            if val <100:
                val += 5

            mset.update({'ip':i},{'$set':{'value':val}})

            print('%s验证成功'%i)
        else:
            val -= 5
            if val<=0:
                mset.delete_one({'ip':i})
            else:
                mset.update({'ip': i}, {'$set': {'value': val}})
            print('%s失效'%i)
    except:
        val -= 5
        if val <= 0:
            mset.delete_one({'ip': i})
        else:
            mset.update({'ip': i}, {'$set': {'value': val}})
        print ('失效%s'%i)
    return 0


if __name__ =='__main__':
    t1 = time.time()
    cishu = 0
    pool = multiprocessing.Pool()
    while 1:#死循环 相当于一直更新代理列表

        finalli = multi(pool)
        fhandle = open('C:/Users/yehuo/PycharmProjects/proxy/a.txt','w',encoding='utf-8')
        fhandle.write(str(finalli))
        fhandle.close()
        yanzheng(mset,pool)
        cishu +=1
        print('已运行%s次'%cishu)
        time.sleep(60)#
    t2 = time.time()
    print(t2-t1)
